{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# stats_act_fn\n",
    "\n",
    "> stats for compare activation functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "baseline - Relu, lr=3e-3\n",
    "[0.628, 0.614, 0.630, 0.610, 0.618, 0.610, 0.626, 0.624, 0.618, 0.592]\n",
    "mean: 0.6170, std: 0.0107\n",
    "\n",
    "Selu, a=1, lr=3e-3\n",
    "[0.602, 0.616, 0.628, 0.608, 0.626, 0.6, 0.590, 0.632, 0.630, 0.608]\n",
    "mean: 0.6140, std: 0.0138\n",
    "\n",
    "Selu,  lr 3e-3\n",
    "[0.62, 0.636, 0.616, 0.606, 0.578, 0.63, 0.618, 0.626, 0.62, 0.626]\n",
    "mean: 0.6176, std: 0.0153\n",
    "\n",
    "Celu\n",
    "[0.65, 0.616, 0.598, 0.622, 0.610, 0.596, 0.590, 0.636, 0.626, 0.622]\n",
    "mean: 0.6166, std: 0.0178\n",
    "\n",
    "Mish\n",
    "[0.586, 0.590, 0.612, 0.608, 0.636, 0.616, 0.6, 0.616, 0.62, 0.646]\n",
    "mean: 0.6130, std: 0.0177\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stat(arr):\n",
    "    print(f\"mean: {arr.mean():0.4f}, std: {arr.std():0.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# base Relu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6200, std: 0.0078\n"
     ]
    }
   ],
   "source": [
    "# lr 3e-3\n",
    "res=np.array([0.628, 0.614, 0.630, 0.610, 0.618])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6140, std: 0.0123\n"
     ]
    }
   ],
   "source": [
    "# lr 3e-3\n",
    "res=np.array([0.610, 0.626, 0.624, 0.618, 0.592])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6170, std: 0.0107\n"
     ]
    }
   ],
   "source": [
    "res=np.array([0.628, 0.614, 0.630, 0.610, 0.618, 0.610, 0.626, 0.624, 0.618, 0.592])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Selu a=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6160, std: 0.0100\n"
     ]
    }
   ],
   "source": [
    "# lr 3e-3\n",
    "res=np.array([0.602, 0.616, 0.628, 0.608, 0.626])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6120, std: 0.0165\n"
     ]
    }
   ],
   "source": [
    "# lr 3e-3\n",
    "res=np.array([0.6, 0.590, 0.632, 0.630, 0.608])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6140, std: 0.0138\n"
     ]
    }
   ],
   "source": [
    "res=np.array([0.602, 0.616, 0.628, 0.608, 0.626, 0.6, 0.590, 0.632, 0.630, 0.608])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5860, std: 0.0000\n"
     ]
    }
   ],
   "source": [
    "# lr 5e-3\n",
    "res=np.array([0.586, ])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lr 5e-3\n",
    "res=np.array([0.624, ])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5800, std: 0.0216\n"
     ]
    }
   ],
   "source": [
    "# lr 1e-2\n",
    "res=np.array([0.550, 0.6, 0.59])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6100, std: 0.0230\n"
     ]
    }
   ],
   "source": [
    "# lr 1e-2\n",
    "res=np.array([0.614, 0.580, 0.636])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5950, std: 0.0269\n"
     ]
    }
   ],
   "source": [
    "res=np.array([0.550, 0.6, 0.59, 0.614, 0.580, 0.636])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "######\n",
    "# Celu with a=0.075"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5844, std: 0.0104\n"
     ]
    }
   ],
   "source": [
    "# lr 1e-3\n",
    "res=np.array([0.582, 0.574, 0.604, 0.584, 0.578])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5916, std: 0.0090\n"
     ]
    }
   ],
   "source": [
    "# lr 1e-3\n",
    "res=np.array([0.578, 0.584, 0.6, 0.6, 0.596])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5880, std: 0.0104\n"
     ]
    }
   ],
   "source": [
    "res=np.array([0.582, 0.574, 0.604, 0.584, 0.578, 0.578, 0.584, 0.6, 0.6, 0.596])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6112, std: 0.0192\n"
     ]
    }
   ],
   "source": [
    "# lr 3e-3\n",
    "res=np.array([0.62, 0.636, 0.616, 0.606, 0.578])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6240, std: 0.0044\n"
     ]
    }
   ],
   "source": [
    "res=np.array([0.63, 0.618, 0.626, 0.62, 0.626])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6176, std: 0.0153\n"
     ]
    }
   ],
   "source": [
    "# lr 3e-3\n",
    "res=np.array([0.62, 0.636, 0.616, 0.606, 0.578, 0.63, 0.618, 0.626, 0.62, 0.626])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6092, std: 0.0116\n"
     ]
    }
   ],
   "source": [
    "# lr 6e-3\n",
    "res=np.array([0.614, 0.61, 0.598, 0.596, 0.628])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5960, std: 0.0157\n"
     ]
    }
   ],
   "source": [
    "# lr 6e-3\n",
    "res=np.array([0.606, 0.590, 0.604, 0.568, 0.612])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6064, std: 0.0179\n"
     ]
    }
   ],
   "source": [
    "# lr 3e-3\n",
    "res=np.array([0.586, 0.590, 0.612, 0.608, 0.636])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6196, std: 0.0149\n"
     ]
    }
   ],
   "source": [
    "# lr 3e-3\n",
    "res=np.array([0.616, 0.6, 0.616, 0.62, 0.646])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6130, std: 0.0177\n"
     ]
    }
   ],
   "source": [
    "res=np.array([0.586, 0.590, 0.612, 0.608, 0.636, 0.616, 0.6, 0.616, 0.62, 0.646])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6064, std: 0.0134\n"
     ]
    }
   ],
   "source": [
    "# lr 6e-3\n",
    "res=np.array([0.616, 0.606, 0.626, 0.590, 0.594])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6092, std: 0.0097\n"
     ]
    }
   ],
   "source": [
    "# lr 6e-3\n",
    "res=np.array([0.622, 0.614, 0.596, 0.614, 0.6])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6078, std: 0.0118\n"
     ]
    }
   ],
   "source": [
    "res=np.array([0.616, 0.606, 0.626, 0.590, 0.594, 0.622, 0.614, 0.596, 0.614, 0.6])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5956, std: 0.0093\n"
     ]
    }
   ],
   "source": [
    "# lr 1e-3\n",
    "res=np.array([0.604, 0.6, 0.580, 0.604, 0.590])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5880, std: 0.0107\n"
     ]
    }
   ],
   "source": [
    "# lr 1e-3\n",
    "res=np.array([0.6, 0.574, 0.590, ])\n",
    "stat(res)\n",
    "##############"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Selu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5840, std: 0.0057\n"
     ]
    }
   ],
   "source": [
    "# lr 1e-3\n",
    "res=np.array([0.586, 0.574, 0.582, 0.588, 0.590])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5852, std: 0.0103\n"
     ]
    }
   ],
   "source": [
    "# lr 1e-3\n",
    "res=np.array([0.584, 0.590, 0.578, 0.572, 0.602])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5846, std: 0.0083\n"
     ]
    }
   ],
   "source": [
    "res=np.array([0.586, 0.574, 0.582, 0.588, 0.590, 0.584, 0.590, 0.578, 0.572, 0.602])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6164, std: 0.0054\n"
     ]
    }
   ],
   "source": [
    "# lr 3e-3\n",
    "res=np.array([0.610, 0.622, 0.622, 0.618, 0.610])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6024, std: 0.0096\n"
     ]
    }
   ],
   "source": [
    "# lr 3e-3\n",
    "res=np.array([0.594, 0.588, 0.608, 0.612, 0.610])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6094, std: 0.0105\n"
     ]
    }
   ],
   "source": [
    "res=np.array([0.610, 0.622, 0.622, 0.618, 0.610, 0.594, 0.588, 0.608, 0.612, 0.610])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6192, std: 0.0173\n"
     ]
    }
   ],
   "source": [
    "# lr 6e-3\n",
    "res=np.array([0.65, 0.616, 0.598, 0.622, 0.610])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6140, std: 0.0178\n"
     ]
    }
   ],
   "source": [
    "# lr 6e-3\n",
    "res=np.array([0.596, 0.590, 0.636, 0.626, 0.622])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6166, std: 0.0178\n"
     ]
    }
   ],
   "source": [
    "res=np.array([0.65, 0.616, 0.598, 0.622, 0.610, 0.596, 0.590, 0.636, 0.626, 0.622])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.6060, std: 0.0291\n"
     ]
    }
   ],
   "source": [
    "# lr 1e-2\n",
    "res=np.array([0.622, 0.618, 0.624, 0.618, 0.548])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5808, std: 0.0368\n"
     ]
    }
   ],
   "source": [
    "# lr 1e-2\n",
    "res=np.array([0.55, 0.526, 0.614, 0.620, 0.594])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 0.5934, std: 0.0355\n"
     ]
    }
   ],
   "source": [
    "res=np.array([0.622, 0.618, 0.624, 0.618, 0.548, 0.55, 0.526, 0.614, 0.620, 0.594])\n",
    "stat(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
